tests.test_eval test_eval_cli

Stack Traces | 31.3s run time

@pytest.mark.slow
    def test_eval_cli():
        runner = CliRunner()
        test_set = ["hello"]
        result = runner.invoke(
            main,
            [
                *test_set,
            ],
        )
        assert result
        assert result.exit_code == 0
>       assert "correct file" in result.output
E       AssertionError: assert 'correct file' in '=== Running evals ===\n=== Completed test hello ===\n=== Completed test hello ===\n=== Completed test hello ===\n\n=== Finished ===\n\n\n\n=== Model Results ===\n\nResults for model: openai/gpt-4o\nCompleted 1 tests in 0.00s:\n- hello in 0.00s (gen: 0.00s, run: 0.00s, eval: 0.00s)\n\nResults for model: anthropic/claude-3-5-sonnet-20240620\nCompleted 1 tests in 0.00s:\n- hello in 0.00s (gen: 0.00s, run: 0.00s, eval: 0.00s)\n\nResults for model: openrouter/meta-llama/llama-3.1-405b-instruct\nCompleted 1 tests in 0.00s:\n- hello in 0.00s (gen: 0.00s, run: 0.00s, eval: 0.00s)\n\n\n=== Model Comparison ===\nModel                                          hello\n---------------------------------------------  --------\nopenai/gpt-4o                                  ❌ 0.00s\nanthropic/claude-3-5-sonnet-20240620           ❌ 0.00s\nopenrouter/meta-llama/llama-3.1-405b-instruct  ❌ 0.00s\n\nResults saved to .../gptme/eval_results/eval_results_20240909_204514.csv\n'
E        +  where '=== Running evals ===\n=== Completed test hello ===\n=== Completed test hello ===\n=== Completed test hello ===\n\n=== Finished ===\n\n\n\n=== Model Results ===\n\nResults for model: openai/gpt-4o\nCompleted 1 tests in 0.00s:\n- hello in 0.00s (gen: 0.00s, run: 0.00s, eval: 0.00s)\n\nResults for model: anthropic/claude-3-5-sonnet-20240620\nCompleted 1 tests in 0.00s:\n- hello in 0.00s (gen: 0.00s, run: 0.00s, eval: 0.00s)\n\nResults for model: openrouter/meta-llama/llama-3.1-405b-instruct\nCompleted 1 tests in 0.00s:\n- hello in 0.00s (gen: 0.00s, run: 0.00s, eval: 0.00s)\n\n\n=== Model Comparison ===\nModel                                          hello\n---------------------------------------------  --------\nopenai/gpt-4o                                  ❌ 0.00s\nanthropic/claude-3-5-sonnet-20240620           ❌ 0.00s\nopenrouter/meta-llama/llama-3.1-405b-instruct  ❌ 0.00s\n\nResults saved to .../gptme/eval_results/eval_results_20240909_204514.csv\n' = <Result okay>.output

tests/test_eval.py:20: AssertionError